
/*
package backup;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.TreeMap;

import modules.*;
import functions.*;


public abstract class DocumentTerms {

	String name;
	Map<String, Double> termCounts;
	Map<String, Double> termFrequencies;

	public String getName()
	{
		return name;
	}

	public void setName(String value)
	{
		name = value;
	}	

	public Map<String,Double> getTermCounts(){
		return termCounts;
	}

	public DocumentTerms(){
		termCounts = new HashMap<String, Double>();
	}
	
	public void ClearTermCounts(){
		if (this.termCounts != null)
			this.termCounts.clear();
		if (this.termFrequencies != null)
			this.termFrequencies.clear();
	}
	
	public void AddTerm(String term, double count){
		if (this.termCounts == null)
			this.termCounts = new HashMap<String, Double>();

		if (!this.termCounts.containsKey(term))
			this.termCounts.put(term, count);
		else
			this.termCounts.put(term, this.termCounts.get(term) + count);
	}
	
	public int NumberOfTerms(){
		if (this.termCounts != null)
			return termCounts.size();
		return 0;
	}
	
	private double GetTotalTermCount(){
		if (termCounts == null)
			return 0;
		double count = 0;
		for (Map.Entry<String, Double> termcount : termCounts.entrySet())
		{
			count += termcount.getValue();
		}
		return count;
	}
	
	private double GetMaxTermCount(){
		if (termCounts == null)
			return 0.0;
		double maxcount = 0.0;
		DoubleValueComparator bvc = new DoubleValueComparator(termCounts);
		TreeMap<String, Double> tMap = new TreeMap<String, Double>(bvc);
		tMap.putAll(termCounts);
		
    	for(Map.Entry<String, Double> termFrequency : tMap.entrySet() ){
			maxcount = termFrequency.getValue();
			break;
    	}
		return maxcount;
	}
	
	public Map<String, Double> GetTF(){
		termFrequencies = new HashMap<String, Double>();
		double totalCount = GetTotalTermCount();
		for (Map.Entry<String, Double> termCount : termCounts.entrySet())
		{
			termFrequencies.put(termCount.getKey(), termCount.getValue() / (double)totalCount);
		}
		return termFrequencies;
	}
	
	public Map<String, Double> GetTF_Norm(){
		termFrequencies = new HashMap<String, Double>();
		double maxcount = GetMaxTermCount();
		for (Map.Entry<String, Double> termCount : termCounts.entrySet())
		{
			termFrequencies.put(termCount.getKey(), termCount.getValue() / (double)maxcount);
		}
		return termFrequencies;
	}

	public void PrintTF(int topK){ 
		if (termFrequencies == null)
			GetTF();
		int i = topK;
		DoubleValueComparator bvc = new DoubleValueComparator(termFrequencies);
		TreeMap<String, Double> tMap = new TreeMap<String, Double>(bvc);
		tMap.putAll(termFrequencies);
		
    	for(Map.Entry<String, Double> termFrequency : tMap.entrySet() ){
    		if( i-- > 0)
    			System.out.println( termFrequency.getKey() + "\t" + termFrequency.getValue() );
    	}
	}
	
	public void RemoveStopwords(StopWords stopwords){
		HashSet<String> stopwordList = stopwords.GetStopWords();
		for(String stopword : stopwordList){
			termCounts.remove(stopword);
		}
	}
	
	public void RemoveDigits(){
		Iterator<Map.Entry<String,Double>> iter = termCounts.entrySet().iterator();
		while (iter.hasNext()) {
		    Map.Entry<String,Double> entry = iter.next();
		    if(isDigit(entry.getKey())){
		        iter.remove();
		    }
		}
	}

    public static boolean isDigit( String input )  
    {  
       try  
       {  
          Integer.parseInt( input );  
          return true;  
       }  
       catch( Exception e)  
       {  
          return false;  
       }  
    }
    
    public static boolean isDouble( String input )  
    {  
       try  
       {  
          Double.parseDouble( input );  
          return true;  
       }  
       catch( Exception e)  
       {  
          return false;  
       }  
    }
    
	public void RemoveUnimportantWords(double threshold){
		if(termFrequencies == null)
			GetTF();
		for(Map.Entry<String, Double> termFrequency : termFrequencies.entrySet()){
			if( termFrequency.getValue() < threshold )
				termCounts.remove(termFrequency.getKey());
		}
	}
	
	public void RemoveUnimportantWords(double threshold, String filename) throws Exception{
		if(termFrequencies == null)
			GetTF();
		for(Map.Entry<String, Double> termFrequency : termFrequencies.entrySet()){
			if( termFrequency.getValue() < threshold ){
				termCounts.remove(termFrequency.getKey());
			}
		}
	}
	
	public void RemoveMentions(){ 
		if(termFrequencies == null)
			GetTF();
		for(Map.Entry<String, Double> termFrequency : termFrequencies.entrySet()){
			if(termFrequency.getKey().startsWith("@") && termFrequency.getKey().length() > 1)
				termCounts.remove(termFrequency.getKey());
		}
	}
	
    public static DocumentTerms ReadDocumentTermCounts(String filename){
    	try{
	    	DocumentTerms document = new Terms();
			FileReader fr = new FileReader(filename);
			BufferedReader reader = new BufferedReader(fr);
			String line = "";
	        while ((line = reader.readLine()) != null)
	        {			
	        	if(line.startsWith("//")){
	        		continue;
	        	}
	        	StringTokenizer tokens = new StringTokenizer (line.toUpperCase(),"\t");
	        	String tk = tokens.nextToken();
	        	String tk2 = tokens.nextToken();

	        	if(!tk.equals("RT") && !tk.startsWith("http")){
	        		if(!DocumentTerms.isDouble(tk))
	        			if(tk.length() != 1 || !(tk.charAt(0) >= 'A' && tk.charAt(0) <= 'Z') ){
	        				document.AddTerm(tk, Double.parseDouble(tk2));
	        			}
	        	}
			}
	        reader.close();
	        fr.close();
	        return document;
    	}catch(Exception e){
    		e.printStackTrace();
    		return null;
    	}
    }
    
    public static DocumentTerms ReadDocumentTermCounts(String file, StopWords stopwords, double frequencyThreshold) {
    	try{
    		String docName = file.substring(file.lastIndexOf('\\') + 1).replaceAll(".txt", "");
    		if(docName.contains("Tweet") || docName.contains("Facebook")){
    			docName = docName.replaceAll(" Tweet WordCount", "").replaceAll(" Facebook WordCount", "");
    		}else{
    			String docArray [] = docName.split(" ");
        		docName = docArray[0] + " "+ docArray[1];
    		}
    		
	        DocumentTerms document = ReadDocumentTermCounts(file);
	        document.setName( docName );
	        document.RemoveStopwords(stopwords);
	        document.RemoveUnimportantWords(frequencyThreshold, document.getName());
	        return document;
	        
    	}catch(Exception e){
    		e.printStackTrace();
    		return null;
    	}
    }
    
    public static DocumentTerms ReadDocumentTermCountsAll(String file, StopWords stopwords, double frequencyThreshold, String cname) {
    	try{
    		System.out.println(file);
    		
	        DocumentTerms document = ReadDocumentTermCounts(file);
	        document.setName( cname +" "+ file.substring(file.lastIndexOf('\\') + 1).replaceAll(" WordCount.txt", "") );
	        document.RemoveStopwords(stopwords);
	      
	        return document;
    	}catch(Exception e){
    		e.printStackTrace();
    		return null;
    	}
    }
    
    public static List<DocumentTerms> GetDocumentWordCounts(String filepath, StopWords stopwords, double frequencyThreshold, String mode){
    	try{
    		List<DocumentTerms> documentList = new ArrayList<DocumentTerms>();
			File dir = new File(filepath);
			File[] listFiles = dir.listFiles();

			for (File file : listFiles) {
				if(file.isFile()){
					if(file.getName().contains(mode))
						documentList.add( ReadDocumentTermCounts( file.getAbsolutePath(), stopwords, frequencyThreshold  ) );
				}
			}
			return documentList;
    	}catch(Exception e){
    		e.printStackTrace();
    		return null;
    	}
    }
    
    public static List<DocumentTerms> GetDocumentWordCountsAll(String filepath, StopWords stopwords, double frequencyThreshold){
    	try{
    		List<DocumentTerms> documentList = new ArrayList<DocumentTerms>();
	        
			File dir = new File(filepath);
			File[] listFiles = dir.listFiles();
			
			List<File> dirCategories  = FileFunction.getListOfSubDirectories(filepath);				
			
			for (File category : dirCategories) {
				List<File> dirSubCategories = FileFunction.getListOfSubDirectories(category.getAbsolutePath());			

				for (File subCategory : dirSubCategories) {	
					
					List<File> newsArticles = FileFunction.getListOfFiles(subCategory.getAbsolutePath());	
					
					for (File newsArticle : newsArticles) {	
						documentList.add( ReadDocumentTermCountsAll( newsArticle.getAbsolutePath(), stopwords, frequencyThreshold, category.getName()+" "+subCategory.getName() ) );						
					}

				}
			}
			
			return documentList;
    	}catch(Exception e){
    		e.printStackTrace();
    		return null;
    	}
    	
    }
    
    
    public static List<DocumentTerms> GetDocumentWordCounts(String username, String filepath, StopWords stopwords, double frequencyThreshold){
    	
    	try{
    		List<DocumentTerms> documentList = new ArrayList<DocumentTerms>();
	        File dir = new File(filepath);
			File[] listFiles = dir.listFiles();
			for (File file : listFiles) {
				if(file.getAbsolutePath().contains(username))
						documentList.add( ReadDocumentTermCounts( file.getAbsolutePath(), stopwords, frequencyThreshold  ) );
				
			}
			return documentList;
    	}catch(Exception e){
    		e.printStackTrace();
    		return null;
    	}
    	
    }

}

*/
